{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c2debc98-af5f-4194-8e6f-223b2bc9ebf5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ff9fc149-5b49-4c71-96f1-1c1494a593d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/azureuser/miniconda3/envs/fundopt/lib/python3.9/site-packages/arctic-1.80.0-py3.9.egg/arctic/store/_pandas_ndarray_store.py:8: FutureWarning: The Panel class is removed from pandas. Accessing it from the top-level namespace will also be removed in the next version\n"
     ]
    }
   ],
   "source": [
    "from arctic import Arctic, CHUNK_STORE\n",
    "lib = Arctic('localhost')\n",
    "fund = lib.get_library('fund')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "88601ff4-bbf7-45cf-b9e0-d37dd7fd2c73",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0c1256d0-5bc8-4219-9fc2-1f110ac6483b",
   "metadata": {},
   "outputs": [],
   "source": [
    "symbols = fund.list_symbols()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "120a32d1-e5c7-4c5e-a46d-76ae603d4bbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_fund_split_data(symbol):\n",
    "    url = f'http://fundf10.eastmoney.com/fhsp_{symbol}.html'\n",
    "    res = requests.get(url)\n",
    "    html = BeautifulSoup(res.text)\n",
    "    split_data = html.find('table', attrs={'class':'w782 comm fhxq'})\n",
    "    split = pd.read_html(str(split_data))[0]\n",
    "    \n",
    "    if split.iloc[0,0] == '暂无拆分信息!':\n",
    "        return pd.DataFrame()\n",
    "    \n",
    "    rename = {'年份' : 'year', \n",
    "              '拆分折算日' : 'date', \n",
    "              '拆分类型' : 'splitType', \n",
    "              '拆分折算比例' : 'amount'}\n",
    "\n",
    "    split = split.rename(columns=rename)\n",
    "    \n",
    "    split['date'] = pd.to_datetime(split['date'])\n",
    "    split['amount'] = split['amount'].str.split(':', expand=True)[1].astype(float)\n",
    "\n",
    "    split = split.drop(['year', 'splitType'], axis=1).set_index('date').sort_index()\n",
    "    \n",
    "    return split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "da98c900-442c-45f7-971f-3de0997cf8f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_fund_div_data(symbol):\n",
    "    \n",
    "    url = 'https://stock.finance.sina.com.cn/fundInfo/api/openapi.php/FundPageInfoService.tabfh'\n",
    "\n",
    "    data_input = {\n",
    "        'symbol' : symbol, \n",
    "        'format' : 'json',\n",
    "    }\n",
    "\n",
    "    resp = requests.get(url, params=data_input)\n",
    "    data = resp.json()\n",
    "    fhdata = data['result']['data']['fhdata']\n",
    "    \n",
    "    if len(fhdata)==0:\n",
    "        return pd.DataFrame()\n",
    "    \n",
    "    div = pd.DataFrame(fhdata).astype({'mffh':float})\n",
    "    \n",
    "    div = div[div['mffh'] > 0.0]\n",
    "    \n",
    "    div = div.drop('fhr', axis=1)\n",
    "\n",
    "    RENAME = {\n",
    "        'djr' : 'date',\n",
    "        'mffh' : 'amount',\n",
    "    }\n",
    "\n",
    "    div = div.rename(columns = RENAME)\n",
    "    div['date'] = pd.to_datetime(div['date'])\n",
    "    # sort div data frame from old to new\n",
    "    div = div.set_index('date').sort_index()\n",
    "    \n",
    "    return div"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "ee3a93f2-f925-4e14-99eb-04ac33e35634",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_fund_adj_data(symbol):\n",
    "    split = get_fund_split_data(symbol).assign(type='split')\n",
    "    div = get_fund_div_data(symbol).assign(type='div')\n",
    "    adj = pd.concat([split, div]).sort_index()\n",
    "    return adj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0d27e6d6-c917-4210-a3be-d92fe1d6d1b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "da29a063-d092-4306-9d10-07bbd2baeba6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1668/1668 [45:08<00:00,  1.62s/it]  \n"
     ]
    }
   ],
   "source": [
    "for symbol in tqdm.tqdm(symbols[2131:]):\n",
    "    adj = get_fund_adj_data(symbol)\n",
    "    adj.to_csv(f'adj_temp/{symbol}.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "2f0b5664-c9d2-47e3-bee3-2be63147264f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "f0fc1e58-47fd-45a5-9dec-0139e773538f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:arctic.chunkstore.chunkstore:Library created, but couldn't enable sharding: no such command: 'enablesharding', full error: {'ok': 0.0, 'errmsg': \"no such command: 'enablesharding'\", 'code': 59, 'codeName': 'CommandNotFound'}. This is OK if you're not 'admin'\n"
     ]
    }
   ],
   "source": [
    "lib.initialize_library('fund_adj', CHUNK_STORE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "9b894e06-21f2-4165-9270-396e8e28590a",
   "metadata": {},
   "outputs": [],
   "source": [
    "fund_adj = lib.get_library('fund_adj')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "57be6e32-4dbd-4b7e-85a9-02969885aea1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Appending symbol 000655 to db\n",
      "Appending symbol 002407 to db\n",
      "Appending symbol 002265 to db\n",
      "Appending symbol 288002 to db\n",
      "Appending symbol 000025 to db\n",
      "Appending symbol 240004 to db\n",
      "Appending symbol 000245 to db\n",
      "Appending symbol 163001 to db\n",
      "Appending symbol 000334 to db\n",
      "Appending symbol 003487 to db\n",
      "Appending symbol 162006 to db\n",
      "Appending symbol 540006 to db\n",
      "Appending symbol 519733 to db\n",
      "Appending symbol 000190 to db\n",
      "Appending symbol 370023 to db\n",
      "Appending symbol 000128 to db\n",
      "Appending symbol 180020 to db\n",
      "Appending symbol 002435 to db\n",
      "Appending symbol 002667 to db\n",
      "Appending symbol 510030 to db\n",
      "Appending symbol 290007 to db\n",
      "Appending symbol 519697 to db\n",
      "Appending symbol 003665 to db\n",
      "Appending symbol 003615 to db\n",
      "Appending symbol 001202 to db\n",
      "Appending symbol 379010 to db\n",
      "Appending symbol 450004 to db\n",
      "Appending symbol 320008 to db\n",
      "Appending symbol 541005 to db\n",
      "Appending symbol 163823 to db\n",
      "Appending symbol 160633 to db\n",
      "Appending symbol 510120 to db\n",
      "Appending symbol 002009 to db\n",
      "Appending symbol 000206 to db\n",
      "Appending symbol 003106 to db\n",
      "Appending symbol 161726 to db\n",
      "Appending symbol 003099 to db\n",
      "Appending symbol 210002 to db\n",
      "Appending symbol 090004 to db\n",
      "Appending symbol 000932 to db\n",
      "Appending symbol 310508 to db\n",
      "Appending symbol 159934 to db\n",
      "Appending symbol 206008 to db\n",
      "Appending symbol 003549 to db\n",
      "Appending symbol 000054 to db\n",
      "Appending symbol 001990 to db\n",
      "Appending symbol 159926 to db\n",
      "Appending symbol 000037 to db\n",
      "Appending symbol 660012 to db\n",
      "Appending symbol 002087 to db\n",
      "Appending symbol 000235 to db\n",
      "Appending symbol 000319 to db\n",
      "Appending symbol 166019 to db\n",
      "Appending symbol 001585 to db\n",
      "Appending symbol 163808 to db\n",
      "Appending symbol 700006 to db\n",
      "Appending symbol 240019 to db\n",
      "Appending symbol 000497 to db\n",
      "Appending symbol 000910 to db\n",
      "Appending symbol 100066 to db\n",
      "Appending symbol 510110 to db\n",
      "Appending symbol 002461 to db\n",
      "Appending symbol 550001 to db\n",
      "Appending symbol 001332 to db\n",
      "Appending symbol 610002 to db\n",
      "Appending symbol 000275 to db\n",
      "Appending symbol 002211 to db\n",
      "Appending symbol 675043 to db\n",
      "Appending symbol 502003 to db\n",
      "Appending symbol 519712 to db\n",
      "Appending symbol 001876 to db\n",
      "Appending symbol 003179 to db\n",
      "Appending symbol 003651 to db\n",
      "Appending symbol 070015 to db\n",
      "Appending symbol 002055 to db\n",
      "Appending symbol 001325 to db\n",
      "Appending symbol 450005 to db\n",
      "Appending symbol 519321 to db\n",
      "Appending symbol 001266 to db\n",
      "Appending symbol 360010 to db\n",
      "Appending symbol 510330 to db\n",
      "Appending symbol 290002 to db\n",
      "Appending symbol 398001 to db\n",
      "Appending symbol 159948 to db\n",
      "Appending symbol 167503 to db\n",
      "Appending symbol 002095 to db\n",
      "Appending symbol 519696 to db\n",
      "Appending symbol 002632 to db\n",
      "Appending symbol 519062 to db\n",
      "Appending symbol 164208 to db\n",
      "Appending symbol 630103 to db\n",
      "Appending symbol 003050 to db\n",
      "Appending symbol 001607 to db\n",
      "Appending symbol 519001 to db\n",
      "Appending symbol 003471 to db\n",
      "Appending symbol 340009 to db\n",
      "Appending symbol 002396 to db\n",
      "Appending symbol 002964 to db\n",
      "Appending symbol 001518 to db\n",
      "Appending symbol 470098 to db\n",
      "Appending symbol 160806 to db\n",
      "Appending symbol 000150 to db\n",
      "Appending symbol 003104 to db\n",
      "Appending symbol 000706 to db\n",
      "Appending symbol 000068 to db\n",
      "Appending symbol 161015 to db\n",
      "Appending symbol 000361 to db\n",
      "Appending symbol 001883 to db\n",
      "Appending symbol 512340 to db\n",
      "Appending symbol 001331 to db\n",
      "Appending symbol 001682 to db\n",
      "Appending symbol 700001 to db\n",
      "Appending symbol 000898 to db\n",
      "Appending symbol 165311 to db\n",
      "Appending symbol 960012 to db\n",
      "Appending symbol 002128 to db\n",
      "Appending symbol 002404 to db\n",
      "Appending symbol 206009 to db\n",
      "Appending symbol 002232 to db\n",
      "Appending symbol 003429 to db\n",
      "Appending symbol 121009 to db\n",
      "Appending symbol 001946 to db\n",
      "Appending symbol 002214 to db\n",
      "Appending symbol 003313 to db\n",
      "Appending symbol 000345 to db\n",
      "Appending symbol 000329 to db\n",
      "Appending symbol 000347 to db\n",
      "Appending symbol 000564 to db\n",
      "Appending symbol 002720 to db\n",
      "Appending symbol 003266 to db\n",
      "Appending symbol 020015 to db\n",
      "Appending symbol 540008 to db\n",
      "Appending symbol 470011 to db\n",
      "Appending symbol 620003 to db\n",
      "Appending symbol 020011 to db\n",
      "Appending symbol 001256 to db\n",
      "Appending symbol 002519 to db\n",
      "Appending symbol 001148 to db\n",
      "Appending symbol 519035 to db\n",
      "Appending symbol 002109 to db\n",
      "Appending symbol 262001 to db\n",
      "Appending symbol 002279 to db\n",
      "Appending symbol 160636 to db\n",
      "Appending symbol 001750 to db\n",
      "Appending symbol 003512 to db\n",
      "Appending symbol 200010 to db\n",
      "Appending symbol 519208 to db\n",
      "Appending symbol 166008 to db\n",
      "Appending symbol 000264 to db\n",
      "Appending symbol 253060 to db\n",
      "Appending symbol 720003 to db\n",
      "Appending symbol 165807 to db\n",
      "Appending symbol 000182 to db\n",
      "Appending symbol 000064 to db\n",
      "Appending symbol 001203 to db\n",
      "Appending symbol 003566 to db\n",
      "Appending symbol 001280 to db\n",
      "Appending symbol 003514 to db\n",
      "Appending symbol 519616 to db\n",
      "Appending symbol 260110 to db\n",
      "Appending symbol 002498 to db\n",
      "Appending symbol 519190 to db\n",
      "Appending symbol 162108 to db\n",
      "Appending symbol 000814 to db\n",
      "Appending symbol 519991 to db\n",
      "Appending symbol 003358 to db\n",
      "Appending symbol 519647 to db\n",
      "Appending symbol 217003 to db\n",
      "Appending symbol 001714 to db\n",
      "Appending symbol 002633 to db\n",
      "Appending symbol 377020 to db\n",
      "Appending symbol 000617 to db\n",
      "Appending symbol 002501 to db\n",
      "Appending symbol 002772 to db\n",
      "Appending symbol 003434 to db\n",
      "Appending symbol 001454 to db\n",
      "Appending symbol 002545 to db\n",
      "Appending symbol 206011 to db\n",
      "Appending symbol 740001 to db\n",
      "Appending symbol 001115 to db\n",
      "Appending symbol 200013 to db\n",
      "Appending symbol 000416 to db\n",
      "Appending symbol 161902 to db\n",
      "Appending symbol 100068 to db\n",
      "Appending symbol 270001 to db\n",
      "Appending symbol 240005 to db\n",
      "Appending symbol 090006 to db\n",
      "Appending symbol 202009 to db\n",
      "Appending symbol 002438 to db\n",
      "Appending symbol 160105 to db\n",
      "Appending symbol 002775 to db\n",
      "Appending symbol 002275 to db\n",
      "Appending symbol 511220 to db\n",
      "Appending symbol 160638 to db\n",
      "Appending symbol 003176 to db\n",
      "Appending symbol 519996 to db\n",
      "Appending symbol 200015 to db\n",
      "Appending symbol 519768 to db\n",
      "Appending symbol 110036 to db\n",
      "Appending symbol 110028 to db\n",
      "Appending symbol 210011 to db\n",
      "Appending symbol 150173 to db\n",
      "Appending symbol 002314 to db\n",
      "Appending symbol 050026 to db\n",
      "Appending symbol 003661 to db\n",
      "Appending symbol 519068 to db\n",
      "Appending symbol 000207 to db\n",
      "Appending symbol 040009 to db\n",
      "Appending symbol 202001 to db\n",
      "Appending symbol 001897 to db\n",
      "Appending symbol 550003 to db\n",
      "Appending symbol 510560 to db\n",
      "Appending symbol 003233 to db\n",
      "Appending symbol 002529 to db\n",
      "Appending symbol 253061 to db\n",
      "Appending symbol 470088 to db\n",
      "Appending symbol 002046 to db\n",
      "Appending symbol 002614 to db\n",
      "Appending symbol 001001 to db\n",
      "Appending symbol 000415 to db\n",
      "Appending symbol 002376 to db\n",
      "Appending symbol 000386 to db\n",
      "Appending symbol 000974 to db\n",
      "Appending symbol 002288 to db\n",
      "Appending symbol 000242 to db\n",
      "Appending symbol 519779 to db\n",
      "Appending symbol 166002 to db\n",
      "Appending symbol 168001 to db\n",
      "Appending symbol 580005 to db\n",
      "Appending symbol 960028 to db\n",
      "Appending symbol 164304 to db\n",
      "Appending symbol 310368 to db\n",
      "Appending symbol 000017 to db\n",
      "Appending symbol 000137 to db\n",
      "Appending symbol 470030 to db\n",
      "Appending symbol 673010 to db\n",
      "Appending symbol 000119 to db\n",
      "Appending symbol 206018 to db\n",
      "Appending symbol 003573 to db\n",
      "Appending symbol 002507 to db\n",
      "Appending symbol 410009 to db\n",
      "Appending symbol 210010 to db\n",
      "Appending symbol 002120 to db\n",
      "Appending symbol 003220 to db\n",
      "Appending symbol 001449 to db\n",
      "Appending symbol 160416 to db\n",
      "Appending symbol 001911 to db\n",
      "Appending symbol 002071 to db\n",
      "Appending symbol 000595 to db\n",
      "Appending symbol 003669 to db\n",
      "Appending symbol 675041 to db\n",
      "Appending symbol 166902 to db\n",
      "Appending symbol 519022 to db\n",
      "Appending symbol 002088 to db\n",
      "Appending symbol 519955 to db\n",
      "Appending symbol 510150 to db\n",
      "Appending symbol 370024 to db\n",
      "Appending symbol 202103 to db\n",
      "Appending symbol 112002 to db\n",
      "Appending symbol 163415 to db\n",
      "Appending symbol 002255 to db\n",
      "Appending symbol 217008 to db\n",
      "Appending symbol 000003 to db\n",
      "Appending symbol 050008 to db\n",
      "Appending symbol 003671 to db\n",
      "Appending symbol 001743 to db\n",
      "Appending symbol 002702 to db\n",
      "Appending symbol 002616 to db\n",
      "Appending symbol 100058 to db\n",
      "Appending symbol 000015 to db\n",
      "Appending symbol 002430 to db\n",
      "Appending symbol 090017 to db\n",
      "Appending symbol 000066 to db\n",
      "Appending symbol 001387 to db\n",
      "Appending symbol 519005 to db\n",
      "Appending symbol 360001 to db\n",
      "Appending symbol 002342 to db\n",
      "Appending symbol 001063 to db\n",
      "Appending symbol 519621 to db\n",
      "Appending symbol 163406 to db\n",
      "Appending symbol 360016 to db\n",
      "Appending symbol 000954 to db\n",
      "Appending symbol 531008 to db\n",
      "Appending symbol 000212 to db\n",
      "Appending symbol 610005 to db\n",
      "Appending symbol 001414 to db\n",
      "Appending symbol 002074 to db\n",
      "Appending symbol 180015 to db\n",
      "Appending symbol 000200 to db\n",
      "Appending symbol 001943 to db\n",
      "Appending symbol 000504 to db\n",
      "Appending symbol 020002 to db\n",
      "Appending symbol 002112 to db\n",
      "Appending symbol 090013 to db\n",
      "Appending symbol 121003 to db\n",
      "Appending symbol 000296 to db\n",
      "Appending symbol 002219 to db\n",
      "Appending symbol 270025 to db\n",
      "Appending symbol 040026 to db\n",
      "Appending symbol 163007 to db\n",
      "Appending symbol 519222 to db\n",
      "Appending symbol 002929 to db\n",
      "Appending symbol 511880 to db\n",
      "Appending symbol 000265 to db\n",
      "Appending symbol 630107 to db\n",
      "Appending symbol 040020 to db\n",
      "Appending symbol 519162 to db\n",
      "Appending symbol 001938 to db\n",
      "Appending symbol 162509 to db\n",
      "Appending symbol 162202 to db\n",
      "Appending symbol 001601 to db\n",
      "Appending symbol 000562 to db\n",
      "Appending symbol 070026 to db\n",
      "Appending symbol 000556 to db\n",
      "Appending symbol 000246 to db\n",
      "Appending symbol 000524 to db\n",
      "Appending symbol 519013 to db\n",
      "Appending symbol 350006 to db\n",
      "Appending symbol 002783 to db\n",
      "Appending symbol 162205 to db\n",
      "Appending symbol 160807 to db\n",
      "Appending symbol 002457 to db\n",
      "Appending symbol 002357 to db\n",
      "Appending symbol 519641 to db\n",
      "Appending symbol 002157 to db\n",
      "Appending symbol 233013 to db\n",
      "Appending symbol 001985 to db\n",
      "Appending symbol 000335 to db\n",
      "Appending symbol 002825 to db\n",
      "Appending symbol 240017 to db\n",
      "Appending symbol 240003 to db\n",
      "Appending symbol 288102 to db\n",
      "Appending symbol 000316 to db\n",
      "Appending symbol 001903 to db\n",
      "Appending symbol 202005 to db\n",
      "Appending symbol 233005 to db\n",
      "Appending symbol 400015 to db\n",
      "Appending symbol 519944 to db\n",
      "Appending symbol 002477 to db\n",
      "Appending symbol 660005 to db\n",
      "Appending symbol 519186 to db\n",
      "Appending symbol 377010 to db\n",
      "Appending symbol 000935 to db\n",
      "Appending symbol 160136 to db\n",
      "Appending symbol 003618 to db\n",
      "Appending symbol 001358 to db\n",
      "Appending symbol 100060 to db\n",
      "Appending symbol 002339 to db\n",
      "Appending symbol 000356 to db\n",
      "Appending symbol 519033 to db\n",
      "Appending symbol 650001 to db\n",
      "Appending symbol 002096 to db\n",
      "Appending symbol 519770 to db\n",
      "Appending symbol 161010 to db\n",
      "Appending symbol 002698 to db\n",
      "Appending symbol 000165 to db\n",
      "Appending symbol 400023 to db\n",
      "Appending symbol 001228 to db\n",
      "Appending symbol 395012 to db\n",
      "Appending symbol 000059 to db\n",
      "Appending symbol 164105 to db\n",
      "Appending symbol 003236 to db\n",
      "Appending symbol 000109 to db\n",
      "Appending symbol 000279 to db\n",
      "Appending symbol 000743 to db\n",
      "Appending symbol 510130 to db\n",
      "Appending symbol 161017 to db\n",
      "Appending symbol 519171 to db\n",
      "Appending symbol 002536 to db\n",
      "Appending symbol 000390 to db\n",
      "Appending symbol 001703 to db\n",
      "Appending symbol 519024 to db\n",
      "Appending symbol 163811 to db\n",
      "Appending symbol 164907 to db\n",
      "Appending symbol 350001 to db\n",
      "Appending symbol 002578 to db\n",
      "Appending symbol 002997 to db\n",
      "Appending symbol 003345 to db\n",
      "Appending symbol 000402 to db\n",
      "Appending symbol 519610 to db\n",
      "Appending symbol 001304 to db\n",
      "Appending symbol 210006 to db\n",
      "Appending symbol 240022 to db\n",
      "Appending symbol 519942 to db\n",
      "Appending symbol 519776 to db\n",
      "Appending symbol 002625 to db\n",
      "Appending symbol 050002 to db\n",
      "Appending symbol 003452 to db\n",
      "Appending symbol 580002 to db\n",
      "Appending symbol 501017 to db\n",
      "Appending symbol 968004 to db\n",
      "Appending symbol 000407 to db\n",
      "Appending symbol 163907 to db\n",
      "Appending symbol 002102 to db\n",
      "Appending symbol 161122 to db\n",
      "Appending symbol 050201 to db\n",
      "Appending symbol 001205 to db\n",
      "Appending symbol 320015 to db\n",
      "Appending symbol 000489 to db\n",
      "Appending symbol 162605 to db\n",
      "Appending symbol 002716 to db\n",
      "Appending symbol 000033 to db\n",
      "Appending symbol 510180 to db\n",
      "Appending symbol 002048 to db\n",
      "Appending symbol 630016 to db\n",
      "Appending symbol 001037 to db\n",
      "Appending symbol 003117 to db\n",
      "Appending symbol 160605 to db\n",
      "Appending symbol 270030 to db\n",
      "Appending symbol 160505 to db\n",
      "Appending symbol 519985 to db\n",
      "Appending symbol 002514 to db\n",
      "Appending symbol 002261 to db\n",
      "Appending symbol 511010 to db\n",
      "Appending symbol 580006 to db\n",
      "Appending symbol 519136 to db\n",
      "Appending symbol 253030 to db\n",
      "Appending symbol 202015 to db\n",
      "Appending symbol 002017 to db\n",
      "Appending symbol 502040 to db\n",
      "Appending symbol 001126 to db\n",
      "Appending symbol 540002 to db\n",
      "Appending symbol 000252 to db\n",
      "Appending symbol 161626 to db\n",
      "Appending symbol 003287 to db\n",
      "Appending symbol 001204 to db\n",
      "Appending symbol 673060 to db\n",
      "Appending symbol 160639 to db\n",
      "Appending symbol 000767 to db\n",
      "Appending symbol 000116 to db\n",
      "Appending symbol 002056 to db\n",
      "Appending symbol 270029 to db\n",
      "Appending symbol 002859 to db\n",
      "Appending symbol 002898 to db\n",
      "Appending symbol 519324 to db\n",
      "Appending symbol 000106 to db\n",
      "Appending symbol 080012 to db\n",
      "Appending symbol 161224 to db\n",
      "Appending symbol 002820 to db\n",
      "Appending symbol 001457 to db\n",
      "Appending symbol 002356 to db\n",
      "Appending symbol 003500 to db\n",
      "Appending symbol 570001 to db\n",
      "Appending symbol 002939 to db\n",
      "Appending symbol 000372 to db\n",
      "Appending symbol 003240 to db\n",
      "Appending symbol 002158 to db\n",
      "Appending symbol 020026 to db\n",
      "Appending symbol 630015 to db\n",
      "Appending symbol 519675 to db\n",
      "Appending symbol 002839 to db\n",
      "Appending symbol 519050 to db\n",
      "Appending symbol 001976 to db\n",
      "Appending symbol 003439 to db\n",
      "Appending symbol 530001 to db\n",
      "Appending symbol 519959 to db\n",
      "Appending symbol 000004 to db\n",
      "Appending symbol 257070 to db\n",
      "Appending symbol 161629 to db\n",
      "Appending symbol 164509 to db\n",
      "Appending symbol 210003 to db\n",
      "Appending symbol 580009 to db\n",
      "Appending symbol 002991 to db\n",
      "Appending symbol 519654 to db\n",
      "Appending symbol 161713 to db\n",
      "Appending symbol 020009 to db\n",
      "Appending symbol 040013 to db\n",
      "Appending symbol 003214 to db\n",
      "Appending symbol 100061 to db\n",
      "Appending symbol 166905 to db\n",
      "Appending symbol 165707 to db\n",
      "Appending symbol 000840 to db\n",
      "Appending symbol 519118 to db\n",
      "Appending symbol 481017 to db\n",
      "Appending symbol 485019 to db\n",
      "Appending symbol 040040 to db\n",
      "Appending symbol 161506 to db\n",
      "Appending symbol 110011 to db\n",
      "Appending symbol 373020 to db\n",
      "Appending symbol 000104 to db\n",
      "Appending symbol 000939 to db\n",
      "Appending symbol 000072 to db\n",
      "Appending symbol 160915 to db\n",
      "Appending symbol 202107 to db\n",
      "Appending symbol 320018 to db\n",
      "Appending symbol 002830 to db\n",
      "Appending symbol 002528 to db\n",
      "Appending symbol 000469 to db\n",
      "Appending symbol 519196 to db\n",
      "Appending symbol 550002 to db\n",
      "Appending symbol 001357 to db\n",
      "Appending symbol 673030 to db\n",
      "Appending symbol 002552 to db\n",
      "Appending symbol 450002 to db\n",
      "Appending symbol 750002 to db\n",
      "Appending symbol 240002 to db\n",
      "Appending symbol 000199 to db\n",
      "Appending symbol 003564 to db\n",
      "Appending symbol 003161 to db\n",
      "Appending symbol 161028 to db\n",
      "Appending symbol 001235 to db\n",
      "Appending symbol 110008 to db\n",
      "Appending symbol 003237 to db\n",
      "Appending symbol 002668 to db\n",
      "Appending symbol 320014 to db\n",
      "Appending symbol 519187 to db\n",
      "Appending symbol 003527 to db\n",
      "Appending symbol 002101 to db\n",
      "Appending symbol 160125 to db\n",
      "Appending symbol 003162 to db\n",
      "Appending symbol 519320 to db\n",
      "Appending symbol 000354 to db\n",
      "Appending symbol 165509 to db\n",
      "Appending symbol 161903 to db\n",
      "Appending symbol 163114 to db\n",
      "Appending symbol 160225 to db\n",
      "Appending symbol 288001 to db\n",
      "Appending symbol 003668 to db\n",
      "Appending symbol 240011 to db\n",
      "Appending symbol 257010 to db\n",
      "Appending symbol 080003 to db\n",
      "Appending symbol 002021 to db\n",
      "Appending symbol 002398 to db\n",
      "Appending symbol 610103 to db\n",
      "Appending symbol 002419 to db\n",
      "Appending symbol 003574 to db\n",
      "Appending symbol 161723 to db\n",
      "Appending symbol 003269 to db\n",
      "Appending symbol 001285 to db\n",
      "Appending symbol 582201 to db\n",
      "Appending symbol 202101 to db\n",
      "Appending symbol 001964 to db\n",
      "Appending symbol 519612 to db\n",
      "Appending symbol 001654 to db\n",
      "Appending symbol 257020 to db\n",
      "Appending symbol 229002 to db\n",
      "Appending symbol 001705 to db\n",
      "Appending symbol 003379 to db\n",
      "Appending symbol 159907 to db\n",
      "Appending symbol 481004 to db\n",
      "Appending symbol 510210 to db\n",
      "Appending symbol 166801 to db\n",
      "Appending symbol 160722 to db\n",
      "Appending symbol 450019 to db\n",
      "Appending symbol 002447 to db\n",
      "Appending symbol 487016 to db\n",
      "Appending symbol 002689 to db\n",
      "Appending symbol 070011 to db\n",
      "Appending symbol 160622 to db\n",
      "Appending symbol 002635 to db\n",
      "Appending symbol 165512 to db\n",
      "Appending symbol 001086 to db\n",
      "Appending symbol 502000 to db\n",
      "Appending symbol 519740 to db\n",
      "Appending symbol 550016 to db\n",
      "Appending symbol 003045 to db\n",
      "Appending symbol 163302 to db\n",
      "Appending symbol 377530 to db\n",
      "Appending symbol 020022 to db\n",
      "Appending symbol 163809 to db\n",
      "Appending symbol 166012 to db\n",
      "Appending symbol 002717 to db\n",
      "Appending symbol 080006 to db\n",
      "Appending symbol 510020 to db\n",
      "Appending symbol 100022 to db\n",
      "Appending symbol 002782 to db\n",
      "Appending symbol 003163 to db\n",
      "Appending symbol 000032 to db\n",
      "Appending symbol 050119 to db\n",
      "Appending symbol 002140 to db\n",
      "Appending symbol 002949 to db\n",
      "Appending symbol 040035 to db\n",
      "Appending symbol 470059 to db\n",
      "Appending symbol 110007 to db\n",
      "Appending symbol 167301 to db\n",
      "Appending symbol 000736 to db\n",
      "Appending symbol 160131 to db\n",
      "Appending symbol 160603 to db\n",
      "Appending symbol 470007 to db\n",
      "Appending symbol 002395 to db\n",
      "Appending symbol 675011 to db\n",
      "Appending symbol 110038 to db\n",
      "Appending symbol 960029 to db\n",
      "Appending symbol 003304 to db\n",
      "Appending symbol 040016 to db\n",
      "Appending symbol 161032 to db\n",
      "Appending symbol 000654 to db\n",
      "Appending symbol 002860 to db\n",
      "Appending symbol 000417 to db\n",
      "Appending symbol 000578 to db\n",
      "Appending symbol 519941 to db\n",
      "Appending symbol 002651 to db\n",
      "Appending symbol 485107 to db\n",
      "Appending symbol 050011 to db\n",
      "Appending symbol 050009 to db\n",
      "Appending symbol 110037 to db\n",
      "Appending symbol 165312 to db\n",
      "Appending symbol 450018 to db\n",
      "Appending symbol 162907 to db\n",
      "Appending symbol 485011 to db\n",
      "Appending symbol 160611 to db\n",
      "Appending symbol 002832 to db\n",
      "Appending symbol 160127 to db\n",
      "Appending symbol 000877 to db\n",
      "Appending symbol 217023 to db\n",
      "Appending symbol 519207 to db\n",
      "Appending symbol 000998 to db\n",
      "Appending symbol 001567 to db\n",
      "Appending symbol 519007 to db\n",
      "Appending symbol 000124 to db\n",
      "Appending symbol 340001 to db\n",
      "Appending symbol 002026 to db\n",
      "Appending symbol 161725 to db\n",
      "Appending symbol 002476 to db\n",
      "Appending symbol 519199 to db\n",
      "Appending symbol 001727 to db\n",
      "Appending symbol 090010 to db\n",
      "Appending symbol 160513 to db\n",
      "Appending symbol 000011 to db\n",
      "Appending symbol 000804 to db\n",
      "Appending symbol 610003 to db\n",
      "Appending symbol 519188 to db\n",
      "Appending symbol 001764 to db\n",
      "Appending symbol 360005 to db\n",
      "Appending symbol 510900 to db\n",
      "Appending symbol 000566 to db\n",
      "Appending symbol 100035 to db\n",
      "Appending symbol 040041 to db\n",
      "Appending symbol 003510 to db\n",
      "Appending symbol 002712 to db\n",
      "Appending symbol 121005 to db\n",
      "Appending symbol 000906 to db\n",
      "Appending symbol 161624 to db\n",
      "Appending symbol 000403 to db\n",
      "Appending symbol 001661 to db\n",
      "Appending symbol 000668 to db\n",
      "Appending symbol 320009 to db\n",
      "Appending symbol 002425 to db\n",
      "Appending symbol 050014 to db\n",
      "Appending symbol 519323 to db\n",
      "Appending symbol 582002 to db\n",
      "Appending symbol 502013 to db\n",
      "Appending symbol 310518 to db\n",
      "Appending symbol 165511 to db\n",
      "Appending symbol 550012 to db\n",
      "Appending symbol 003672 to db\n",
      "Appending symbol 450009 to db\n",
      "Appending symbol 000616 to db\n",
      "Appending symbol 003529 to db\n",
      "Appending symbol 002899 to db\n",
      "Appending symbol 160322 to db\n",
      "Appending symbol 002796 to db\n",
      "Appending symbol 003142 to db\n",
      "Appending symbol 610008 to db\n",
      "Appending symbol 002928 to db\n",
      "Appending symbol 000572 to db\n",
      "Appending symbol 519011 to db\n",
      "Appending symbol 001405 to db\n",
      "Appending symbol 110009 to db\n",
      "Appending symbol 519209 to db\n",
      "Appending symbol 151001 to db\n",
      "Appending symbol 100038 to db\n"
     ]
    }
   ],
   "source": [
    "adj_path = 'adj_temp/'\n",
    "for csv in os.listdir(adj_path):\n",
    "    if not csv.endswith('.csv'): \n",
    "        continue\n",
    "\n",
    "    symbol = csv.split('.')[0]\n",
    "    if fund_adj.has_symbol(symbol): \n",
    "        continue\n",
    "    df = pd.read_csv(  os.path.join( adj_path, csv ), index_col=0, parse_dates=True )\n",
    "    df.index.name = 'date'\n",
    "    if not df.empty:\n",
    "        print( 'Appending symbol {} to db'.format( symbol ) )\n",
    "        fund_adj.append( symbol, df, upsert=True, chunk_size = 'M' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "3b6707a3-4521-45c9-8144-34c075c2b429",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2391"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(fund_adj.list_symbols())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "99689ab9-1fa2-4c97-944c-910239cf4cc8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fund.has_symbol('000002')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "b02bcdfd-f58f-419c-af49-112bad508514",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "000014\n"
     ]
    }
   ],
   "source": [
    "for symbol in fund.list_symbols():\n",
    "    if not fund_adj.has_symbol(symbol):\n",
    "        print(symbol)\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "819c3b99-9407-40c1-bf3a-d658075d22cc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
